import Core.MongoDB as MongoDB
import Core.Portfolio as Portfolio
import Core.Gadget as Gadget
import Core.IO as IO
import pymongo
import datetime
import math
import copy
import pandas as pd
import numpy as np
import statsmodels.api as sm
import matplotlib.pyplot as plt
import Core.Quote as Quote
import Analysis.General as General
import Factors
import Factors.FactorContent
import Analysis
import Analysis.RegressionAnalysis

#Test Portfolio

def TestPortfolio(database):

    dt = datetime.datetime(2017,5,1)
    stdDateTime = Gadget.ToUTCDateTime(dt)
    instruments = database.findWithFilter("Instruments", "Stock")

    #---Build Portfolio---
    Portfolio.Create(database,"TestPortfolio",dt, 1000000, "Factor")


    #---Add Trade---

    for i in range(10):
        instrument = instruments[i]
        symbol = instrument["Symbol"]
        quote = Quote.GetQuote(database, symbol, stdDateTime)
        #
        trade = {}
        trade["Symbol"] = symbol
        trade["Price"] = quote["Close"]
        trade["Side"] = "Buy"
        trade["Qty"] = int(1000000 / trade["Price"])
        Portfolio.AddFill(database, "TestPortfolio", trade, stdDateTime)


#Fama-French-3
def FamaFrench3_PrepareData(database):
    for year in range(2006, 2017 + 1):
        dt = datetime.datetime(year, 5, 1)
        stdDateTime = Gadget.ToUTCDateTime(dt)
        #print(dt)

        #---
        fields = ['Symbol', 'BookToMarket', 'Size', "Close"]
        data = []

        #---Loop Instruments---
        filter = {}
        #filter = {"limit":100}
        instruments = database.findWithFilter("Instruments", "Stock", filter)
        start = False
        logContent = ""
        for instrument in instruments:

            symbol = instrument["Symbol"]
            #
            #if symbol == "000409.SZ":
            #    start = True
            #if start == False:
            #    continue

            #
            if instrument["IPO_Date"] > stdDateTime:
                print("NotIPOyet " + symbol)
                logContent += "NotIPOyet," + symbol + "\n"
                continue

            #---Statement @ last Year---
            #---Book to market 用去年年报---
            reportDateTimeShort = str(year-1) + "-12-31"
            reportDateTime = str(year-1) + "-12-31 00:00:00.000"
            statement = database.findWithFilter("Fundamental",symbol + "_Fundamental", {"ReportDate":reportDateTime})
            if len(statement) == 0:
                print("Can'tFindStatement " + symbol)
                logContent += "Can'tFindStatement," + symbol + "\n"
                continue
            statement = statement[0]

            if "TotalEquity" not in statement["Values"]:
                print("MissingTotalEquity " + symbol)
                logContent += "MissingTotalEquity," + symbol + "\n"
                continue
            if statement["Values"]["TotalEquity"] == 0:
                print("ZeroEquity " + symbol)
                logContent += "ZeroEquity," + symbol + "\n"
                continue
            if statement["Values"]["TotalEquity"] < 0:
                print("NegtiveEquity " + symbol)
                logContent += "NegtiveEquity," + symbol + "\n"
                continue
            if "Close" not in statement["Values"]:
                print("MissingClose " + symbol)
                logContent += "MissingClose," + symbol + "\n"
                continue
            if statement["Values"]["TotalShares"] == 0:
                print("ZeroTotalShares " + symbol)
                logContent += "ZeroTotalShares," + symbol + "\n"
                continue
            if statement["Values"]["Close"] == 0:
                print("ZeroClose " + symbol)
                logContent += "ZeroClose," + symbol + "\n"
                continue

            book = statement["Values"]["TotalEquity"]
            market = statement["Values"]["TotalShares"] * statement["Values"]["Close"]
            bookToMarket = book / market

            #---Cap 用最新市值---
            quote = Quote.GetQuote(database, symbol, stdDateTime)
            if quote == None:
                print("Can'tFindQuote " + symbol)
                logContent += "Can'tFindQuote," + symbol + "\n"
                continue
            size = quote["Close"] * quote["Values"]["TotalShares"]
            close = quote["Close"]

            #---Build DataFrames---
            data.append([symbol, bookToMarket, size, close])

        # ---Build DataFrames---
        df = pd.DataFrame(data, columns=fields)
        print(df)
        df.to_csv("D://Data//FamaFrench3//FamaFrench3_" + reportDateTimeShort + ".csv")
        IO.WriteToFile("D://Data//FamaFrench3//FamaFrench3_Log_"  + reportDateTimeShort + ".csv", logContent)
        kkwood = 0


def FamaFrench3_BuildModel(database):

    # ---Build Portfolio---
    beginDateTime = datetime.datetime(2006, 4, 25,15,0,0)
    endingDateTime = datetime.datetime(2017, 9, 1,15,0,0)
    #stdStartDateTime = Gadget.ToUTCDateTime(beginDateTime)
    #stdEndingDateTime = Gadget.ToUTCDateTime(endingDateTime)
    datetime1 = beginDateTime
    datetime2 = datetime.datetime(2006, 5, 1,15,0,0)

    #---Building Portfolio---
    Portfolio.Create(database, "SmallValue", beginDateTime, 10000000, "Factor")
    #Portfolio.Create(database, "BigValue", beginDateTime, 10000000, "Factor")
    #Portfolio.Create(database, "SmallNeutral", beginDateTime, 10000000, "Factor")
    #Portfolio.Create(database, "BigNeutral", beginDateTime, 10000000, "Factor")
    #Portfolio.Create(database, "SmallGrowth", beginDateTime, 10000000, "Factor")
    #Portfolio.Create(database, "BigGrowth", beginDateTime, 10000000, "Factor")

    while datetime2 < endingDateTime:
        stdDateTime1 = Gadget.ToUTCDateTime(datetime1)
        stdDateTime2 = Gadget.ToUTCDateTime(datetime2)
        bmBars = database.getDataSeries("000300.SH_Time_86400_Bar","Index", stdDateTime1, stdDateTime2)
        tradeDate = Gadget.ToLocalDateTime(bmBars.DateTime2())

        #
        print(str(datetime1) + " --- " + str(datetime2))
        print(" Reblance " + str(bmBars.DateTime2()))

        #
        reportDateTimeShort = str(datetime2.year - 1) + "-12-31"
        dfData = pd.read_csv("D://Data//FamaFrench3//FamaFrench3_" + reportDateTimeShort + ".csv")

        #dfData.sort(["BookToMarket"], ascending = True)

        rank70 = dfData.quantile(0.7)
        rank50 = dfData.quantile(0.5)
        rank30 = dfData.quantile(0.3)

        smaValue = []
        bigValue = []
        smaNeutral = []
        bigNeutral = []
        smaGrowth = []
        bigGrowth = []
        for index, row in dfData.iterrows():
            bookToMarket = row["BookToMarket"]
            size =  row["Size"]
            trade = {}
            trade["Symbol"] = row["Symbol"]
            trade["Price"] = row["Close"]
            trade["Side"] = "Buy"
            trade["Weight"] = 0


            #The BE/ME breakpoints are the 30th and 70th NYSE percentiles.
            if bookToMarket >= rank70["BookToMarket"]:
                if size >= rank50["Size"]:
                    bigValue.append(trade)
                else:
                    smaValue.append(trade)
            #
            elif bookToMarket >= rank30["BookToMarket"]:
                if size >= rank50["Size"]:
                    bigNeutral.append(trade)
                else:
                    smaNeutral.append(trade)
            #
            else:
                if size >= rank50["Size"]:
                    bigGrowth.append(trade)
                else:
                    smaGrowth.append(trade)
        #---Loop Data Entry---
        kkwood = 1

        BuildEquialWeightPortfolio(smaValue)
        #Portfolio.CalculateDaily(database,"SmallValue",datetime1, tradeDate)
        Portfolio.Rebalance(database,"SmallValue",smaValue, tradeDate)
        Portfolio.CalculateDaily(database, "SmallValue", tradeDate, tradeDate)

        kkwood = 0
        datetime1 = datetime2
        datetime2 = datetime2.replace(year = datetime2.year + 1)

    #---Loop Year---
    datetime2 = endingDateTime


def BuildEquialWeightPortfolio(trades):
    count = len(trades)
    weight = round(1.0 / count , 6)
    for trade in trades:
        trade["Weight"] = weight
    kkwood = 0


def ProcessFactor2(database, datetime1, factorName, params):
    #
    df = Analysis.General.Profile(database, datetime2, [factorName])

    # ---General Process---
    Factors.FactorContent.ProcessFactors(df)
    return df


def ProcessFactors(database, datetime1, factorName, params):
    print("ProcessFactor", factorName, datetime1)
    top = []
    bottom =[]
    #
    if "LoadDataSeries" in params:
        loadDataSeries = params["LoadDataSeries"]
    else:
        loadDataSeries = False

    isQuarterly = False
    if "IsQuarterly" in params:
        isQuarterly = params["IsQuarterly"]

    # ---Load Factor Data---
    filter = {}
    # ---DataSeries数据，得到距离时间最近因子数值进行排序---
    if loadDataSeries:
        datetime0 = datetime1 + datetime.timedelta(days=-366)  # 最多回溯一年指标
        filter["StdDateTime"] = {"$gte": datetime0, "$lte": datetime1}
        factor = database.Find("Factor", factorName, filter, sort=[("StdDateTime", 1)])
        #
        dfBySymbols = {}
        dataBySymbol = {}

        # ---分类Factor，按照symbol---
        for f in factor:
            symbol = f["Symbol"]
            value = f["Value"]
            if symbol not in dataBySymbol.keys():
                dataBySymbol[symbol] = []
            dataBySymbol[symbol].append(symbol, value)

        # ---Get the last one Build Dataframe---
        data = []
        for symbol, values in dataBySymbol.items():
            data.append(values[-1])  #
        df = pd.DataFrame(data, columns=["Symbol", "Value"])
    #
    elif isQuarterly:
        # 针对季报 “报告时间” 进行取值
        targetReportDate = None
        if datetime1.month == 5:
            targetReportDate = datetime.datetime(datetime1.year-1, 12, 31)
        elif datetime1.month == 9:
            targetReportDate = datetime.datetime(datetime1.year, 6, 30)
        elif datetime1.month == 11:
            targetReportDate = datetime.datetime(datetime1.year, 9, 30)

        filter["ReportDate"] = targetReportDate
        factor = database.Find("Factor", factorName, filter)
        #
        data = []
        for f in factor:
            data.append([f["Symbol"], f["Value"]])
        df = pd.DataFrame(data, columns=["Symbol", "Value"])

    else:
        # ---单一时间，Profile剖面数据，排序后得到list---
        filter["StdDateTime"] = datetime1
        factor = database.Find("Factor", factorName, filter)
        #
        data = []
        for f in factor:
            data.append([f["Symbol"], f["Value"]])
        df = pd.DataFrame(data, columns=["Symbol", "Value"])

    # ---Exclude New Stock---
    dfSymbols = params["ListedSymbols"]
    df = pd.merge(dfSymbols, df, on='Symbol', how='left')

    # ---Temp Print 输出全量数据---
    df.sort_values(by="Value", ascending=True, inplace=True)
    print(df)
    outputFolder = "D:/Data/FactorModel/" + factorName + "/"
    df.to_csv(outputFolder + "factor_" + Gadget.ToDateString(datetime1) + ".csv")

    # ---General Process---
    df.replace([np.inf, -np.inf], np.nan, inplace=True)
    df.dropna(inplace=True)
    notNegtive = False
    assending = True
    percentage = 0.01

    l = len(df)
    if l == 0:
        return df

    # ---Cut Tail---
    qt_min = df.quantile(0.01).values[0]  # 5%分位数
    qt_max = df.quantile(0.99).values[0]  # 95%分位数
    df = df[(df["Value"] > qt_min) & (df["Value"] < qt_max)]

    # ---Special Process---
    if "NonNegative" in params:
        df = df[df["Value"] > 0]

    # ---Sorting---
    df.sort_values(by="Value", ascending=assending, inplace=True)
    # print(df)

    # ---Selection Top / Bottom---
    #n = int(percentage * df.shape[0])
    #bottom = df.head(n)["Symbol"].tolist()
    #top = df.tail(n)["Symbol"].tolist()
    # return bottom, top
    return df


def BuildModel_MultiFators_10Factors(database, datetime1, datetime2):

    factorNames = []
    # Profit
    factorNames.append("ROE_NetIncome2")

    # Valuation
    factorNames.append("PB_LF")
    factorNames.append("PE_TTM")
    factorNames.append("PE_LYR")

    # Cash flow
    factorNames.append("CashROIC")

    # Growth
    factorNames.append("Growth_Revenue_CAGR")
    factorNames.append("Earning_CAGR")

    # Capital
    factorNames.append("Leverage")

    # Opreation
    factorNames.append("AssetTurnover")

    # Price Momentum
    factorNames.append("Momentum20D")

    # Size
    factorNames.append("Cap")

    # Volatility
    factorNames.append("Volatility_240D")

    # Misc
    factorNames.append("Liquidity20D")
    factorNames.append("HolderCR10")
    factorNames.append("Turnover20D")

    #
    factorParam = Factors.FactorParams()
    factors = {}
    for f in factorNames:
        factors[f] = factorParam.get(f)

    #
    df = BuildModel_Multifactors(database, datetime1, datetime2, factors)
    return df


def BuildModel_MultiFators_TotalFactors(database, datetime1, datetime2, save=False, update=False):
    #
    factors = Factors.FactorParams()
    #
    df = BuildModel_Multifactors(database, datetime1, datetime2, factors, save=save, update=update)
    return df


def BuildModel_FamaFrench3(database, datetime1, datetime2, update=False, save=False):
    #
    factors = {}
    factors["Cap"] = {"GroupType": "HiLo"}
    factors["PB_LF"] = {"GroupType": "HiLo"}
    #
    df = BuildModel_Multifactors(database, datetime1, datetime2, factors, market="000001.SH", update=update, save=save)
    #
    return df


def BuildModel_Multifactors_Hibernate(database, datetime1, datetime2, factors, name, update=False):
    #
    if update:
        modelsValue = database.Find("Misc", "Model", {"Name": name}, sort=[("DateTime", 1)])
        if len(modelsValue) > 0:
            m = modelsValue[-1]
            datetime1 = m["DateTime"]

    df = BuildModel_Multifactors(database, datetime1, datetime2, factors)
    print(df)
    SaveModel(df, name)
    #
    return df


def SaveModel(df, name):
    #
    print(df.head())

    fileds = df.columns.values
    #
    for index, row in df.iterrows():
        for field in fileds:
            if field == "DateTime":
                continue
            doc = {}
            doc["Factor"] = field
            doc["DateTime"] = row["DateTime"]
            doc["Value"] = row[field]
            doc["Params"] = "HiLo"
            database.Upsert("Misc", "Model",
                            {"DateTime": doc["DateTime"], "Factor": doc["Factor"], "Params": doc["Params"]},
                            doc)


def LoadModel(database, datetime1, datetime2, factors):

    def LoadGroup(database, group):
        filter["Group"] = group
        sort = [("DateTime",1)]
        data = []
        factorHiValues = database.Find("Misc", "FactorPortfolio", filter, sort)
        for value in factorHiValues:
            data.append([value["DateTime"].date(), value["Value"]])
        #
        df = pd.DataFrame(data, columns=["DateTime", "Value"])
        #
        return df

    # ---Market Beta---
    bmSymbol = "000001.SH"
    dfMarket = General.ReturnsDataSeries_Dataframe(database, bmSymbol, datetime1, datetime2)
    dfMarket.rename(columns={bmSymbol: "Market"}, inplace=True)
    df = dfMarket
    # print(df)

    for factor in factors:
        filter = {}
        filter["Factor"] = factor
        filter["DateTime"] = {"$gte": datetime1, "$lte": datetime2}
        filter["Group"] = "HiLo_0"

        dfLo = LoadGroup(database, "HiLo_0")
        # print(dfLo)
        dfHi = LoadGroup(database, "HiLo_2")
        # print(dfHi)

        #
        dfLo.rename(columns={"Value": "Lo"}, inplace=True)
        dfHi.rename(columns={"Value": "Hi"}, inplace=True)
        dfTemp = pd.merge(dfLo, dfHi, how="left", on="DateTime")
        dfTemp[factor] = dfTemp["Hi"] - dfTemp["Lo"]
        dfTemp = dfTemp[["DateTime", factor]]
        # df.drop(columns=["Hi","Lo"], inplace=True)
        # print(df)
        if df.empty:
            df = dfTemp
        else:
            df = pd.merge(df, dfTemp, how="left", on="DateTime")
    #
    return df

#
def GroupFactorPortfolio(dfFactor, groupType, factorName, params={}):
    #
    dfFactor.sort_values(by=factorName, ascending=False, inplace=True)
    #
    length = dfFactor.shape[0]
    group = []
    #
    if groupType == "Percentile":
        for i in range(10):
            start = int(i * 0.1 * length)
            end = int((i+1) * 0.1 * length)
            if i == 0:
                start = 0
            group.append(dfFactor[start:end])
            #
            # print(group[i].shape[0])
            # print("Start", start, "End", end, "Length", group[i].shape[0], length)
            # print(group[i].head())
            # print("---")
            # print(group[i].tail())
    #
    elif groupType == "Quartile":
        for i in range(4):
            start = int(i * 0.25 * length)
            end = int((i+1) * 0.25 * length)
            if i == 0:
                start = 0
            group.append(dfFactor[start:end])
            #
            # print(group[i])
            # print(start, end, end - start + 1, length)
    #
    elif groupType == "HiLo":
        p30 = int(0.3 * length)
        p70 = int(0.7 * length)
        group.append(dfFactor[0: p30])
        group.append(dfFactor[p30: length-p30])
        group.append(dfFactor[length-p30: length])
        #
        # print(p70)
        # print(length-p30)

        # print(group[i])
        # print(group[0].shape[0], "begin", group[0].head(1), "end", group[0].tail(3))
        # print(group[1].shape[0])
        # print(group[2].shape[0])
    #
    return group


def GenerateFactorPortfolio():
    pass

def UpdateSymbolGroupLocator(factors):
    #
    symbolGroupLocator = {}
    for factorName, values in factors.items():
        groups = values["Portfolio"]
        for i in range(len(groups)):
            group = groups[i]
            for symbol in group:
                location = symbolGroupLocator.get(symbol)
                if location == None:
                    symbolGroupLocator[symbol] = {}
                #
                symbolGroupLocator[symbol][factorName] = i
    #
    return symbolGroupLocator


def ProcessReturns(database, factors, datetime1):
    #
    factorNames = list(factors.keys())

    # ---Find Total "Return Values" at a Specific Day---
    returnByFactors = {}
    for factorName in factorNames:
        groups = factors[factorName]["Group"]
        returnByFactors[factorName] = []
        for iGroup in range(len(groups)):
            returnByFactors[factorName].append([])

    # gte = {"Value": {"$gte": -0.11}}
    # lte = {"Value": {"$lte": 0.11}}
    # filter = {"$and": [gte, lte]}
    filter = {}
    # filter = {"limit": 100}
    filter["DateTime"] = datetime1
    returns = database.Find("Factor", "DailyReturn", filter)

    dataReturns = []
    # ---Process Return---
    for r in returns:
        # ---排除新股---
        symbol = r["Symbol"]
        value = r["Value"]
        dataReturns.append([symbol, value])

        #if symbol not in listedSymbols:
        #    continue

        # ---排除不正常收益---
        # if value > 0.11 or value < -0.11:
        #     continue

        # ---Save Ret to Cache---
        # Fast locating
        # location = symbolGroupLocator.get(symbol)
        # if location == None:
        #     pass
        # else:
        #     for factorName, iGroup in location.items():
        #         returnByFactors[factorName][iGroup].append(value)

        # ---Save Ret to Cache---
        # Old Fashion
        # for factorName in factorNames:
        #     for i in range(len(factors[factorName]["Group"])):
        #         if symbol in factors[factorName]["Group"][i]:
        #             returnByFactors[factorName][i].append(value)
    #
    dfReturns = pd.DataFrame(dataReturns, columns=["Symbol", "Return"])
    return dfReturns


def ProcessFactorPortfolioReturn(database, factors):
    pass


def ReturnsToDataframe(database, datetime1):
    #
    filter = {}
    filter["DateTime"] = datetime1
    returns = database.Find("Factor", "DailyReturn", filter)
    #
    dataReturns = []
    # ---Process Return---
    for r in returns:
        # ---排除新股---
        symbol = r["Symbol"]
        value = r["Value"]
        dataReturns.append([symbol, value])
    #
    dfReturns = pd.DataFrame(dataReturns, columns=["Symbol", "Return"])
    return dfReturns

def DataSeriesToDataframe(dataSeries, fieldname="Return"):
    #
    data = []
    for d in dataSeries:
        symbol = d["DateTime"]
        value = d["Return"]
        data.append([symbol, value])
    #
    dfReturns = pd.DataFrame(data, columns=["DateTime", fieldname])
    return dfReturns


def SaveGroupReturn(database, factorName, groupType, groupIndex, datetime1, value, count):
    target = {"Factor": factorName, "DateTime": Gadget.ToDate(datetime1), "Group": groupType + "_" + str(groupIndex)}
    doc = copy.deepcopy(target)
    doc["Value"] = value
    doc["Count"] = count
    # doc["InstrumentList"] = factors[factorName]["Group"][iGroup]
    database.Upsert("Misc", "FactorPortfolio", target, doc)


# ---实际上在完成回测---
def BuildFactorPortfolio(database, datetime1, datetime2, factorParams, save=False, output=False):

    # 不记录DateTime1，记录DateTime2
    def CalculateRangeReturn(dfReturnCache, portfolios, datetime1, datetime2):
        #
        # print("Calculate RangeReturn", datetime1, datetime2)
        #
        if datetime1 == datetime2:
            return

        for pfNum in range(len(portfolios)):
            portfolio = portfolios[pfNum]
            dfSymbols = portfolio["Symbols"]

            # ---Merge Symbols (May Missing Returns in Cache)---
            selectedSymbols = []
            #
            if dfSymbols.empty:
                continue
            #
            symbols = dfSymbols["Symbol"].tolist()
            for symbol in symbols:
                if symbol in dfReturnCache.columns:
                    selectedSymbols.append(symbol)
            dfStockReturns = dfReturnCache[selectedSymbols]
            # print(dfStockReturns.head())
            dfStockReturns = dfStockReturns.loc[datetime1:datetime2, :]

            # 不记录调仓当天的绩效
            if datetime1 in dfStockReturns.index:
                dfStockReturns.drop(index=[datetime1], inplace=True)

            # print(dfStockReturns.head())
            # dfStockReturns.to_csv("d://temp1.csv")

            # ---Calculate NetValue---
            dfNetValue = pd.DataFrame.cumprod(dfStockReturns + 1)
            # print(dfNetValue.head())
            if pfNum == 0:
                # dfNetValue.to_csv("d://NetValue_" + Gadget.ToDateString(datetime2) + ".csv")
                # print(dfNetValue)
                # a = dfNetValue["Return"].mean()
                # print(datetime2, a)
                pass

            if len(dfNetValue) == 0:
                a = 0

            dfRangeReturns = dfNetValue.iloc[[-1]].T
            # print(dfRangeReturns.head())
            #
            c = dfRangeReturns.columns
            fieldname = c[0]
            dfRangeReturns.rename(columns={fieldname: "NetValue"}, inplace=True)
            #
            dfRangeReturns["RangeReturn"] = dfRangeReturns["NetValue"] - 1
            dfRangeReturns["DateTime1"] = datetime1.date()
            dfRangeReturns["DateTime2"] = datetime2.date()
            days = (datetime2.date() - datetime1.date()).days
            dfRangeReturns["Annualized"] = dfRangeReturns["RangeReturn"] / days * 240
            dfRangeReturns.sort_values(by="RangeReturn", ascending=False, inplace=True)
            # print(dfRangeReturns.head())

            #if pfNum == 0:
            #    dfRangeReturns.to_csv("d://RangeReturn_" + Gadget.ToDateString(datetime2) + ".csv")

            #
            if not portfolio.get("RangeReturn").empty:
                portfolio["RangeReturn"] = pd.concat([portfolio["RangeReturn"], dfRangeReturns])
            else:
                portfolio["RangeReturn"] = dfRangeReturns

            #

            pass


    def Initialize_Portfolio(factors, factorName, groupType):
        groupCount = 0
        if groupType == "Percentile":
            groupCount = 10
        elif groupType == "Quartile":
            groupCount = 4
        elif groupType == "HiLo":
            groupCount = 3
        #
        factors[factorName]["Portfolio"] = []
        for i in range(groupCount):
            newPortfolio = {}
            newPortfolio["Symbols"] = pd.DataFrame()
            newPortfolio["DataSeries"] = []
            newPortfolio["RangeReturn"] = pd.DataFrame()
            #
            factors[factorName]["Portfolio"].append(newPortfolio)
        #
        factors[factorName]["RebalanceDate"] = None


    def Rebalance_Portfolio(factors, factorName, dfFactor, groupType, datetime2, output=False):

        # ---Build Group Portfolio--- 用因子排序分组
        groups = GroupFactorPortfolio(dfFactor, groupType, factorName, factors[factorName])

        # ---Update Portfolio Symbols---
        for groupNum in range(len(groups)):
            dfGroup = groups[groupNum][["Symbol"]]
            factors[factorName]["Portfolio"][groupNum]["Symbols"] = dfGroup
            # factors[factorName]["Portfolio"][groupNum]["Symbols"] = groups[groupNum]["Symbol"].tolist()
            if output:
                dfGroup.to_csv(outputFolder + "Portfolio_" + str(groupNum) + "_" + Gadget.ToDateString(datetime2) + ".csv")

        # ---Update Rebalance Date---
        factors[factorName]["RebalanceDate"] = datetime2


    def UpdateValue_Portfolio(data, dfReturns, datetime2):
        #
        portfolio = data.get("Portfolio")
        #
        if portfolio == None:
            return
        # print(factorName)
        for pfNum in range(len(portfolio)):
            pf = portfolio[pfNum]
            if len(pf["Symbols"]) == 0:
                # print("Null Portfolio", factorName)
                continue
            dfPfReturns = pd.merge(pf["Symbols"], dfReturns, how="inner", on="Symbol")
            # if pfNum == 0:
            #     dfPfReturns.to_csv("d://pf0_returns_" + Gadget.ToDateString(datetime2) + ".csv")
            pfReturn = dfPfReturns["Return"].mean()
            pf["DataSeries"].append([datetime2, pfReturn])


    def Export_Portfolio_NetAsset(dataByFactorName, datetime1, datetime2):
        #
        for factorName, value in dataByFactorName.items():
            df = pd.DataFrame()
            # ---Loop and Merge
            if value.get("Portfolio"):
                #
                for i in range(len(value["Portfolio"])):
                    dataSeries0 = value["Portfolio"][i]["DataSeries"]
                    dfTmp = pd.DataFrame(dataSeries0, columns=["DateTime", "Return" + str(i)])
                    dfTmp["NetValue" + str(i)] = pd.DataFrame.cumprod(dfTmp["Return" + str(i)] + 1)
                    #
                    if df.empty:
                        df = dfTmp[["DateTime", "NetValue" + str(i)]]
                    else:
                        df = pd.merge(df, dfTmp[["DateTime", "NetValue" + str(i)]], on="DateTime", how="left")
                #
                # print(df)
        #
        pathName = "d://data//FactorAnalysis//"
        df.to_csv(pathName + factorName + "//" + "FactorPortfolioReturn_" + Gadget.ToDateString(
            datetime1) + "_" + Gadget.ToDateString(datetime2) + ".csv")


    def Save_Portfolio(database, factors):
        #
        for factorName, value in factors.items():
            params = value
            groupType = params.get("GroupType")
            if groupType == None:
                groupType = "Percentile"

            print("Save Factor Portfolio", factorName)
            portfolios = value["Portfolio"]
            #
            dataToSave = []
            for i in range(len(portfolios)):
                dataSeries = portfolios[i]["DataSeries"]
                #
                doc = {}
                doc["FactorName"] = factorName
                doc["GroupType"] = groupType
                doc["Number"] = i
                #
                for element in dataSeries:
                    newDoc = copy.deepcopy(doc)
                    newDoc["DateTime"] = element[0]
                    newDoc["Date"] = element[0].date()
                    if np.isnan(element[1]):
                        continue
                    newDoc["Value"] = element[1]
                    newDoc["Key2"] = factorName + "_" + groupType + "_" + str(i) + "_" + Gadget.ToDateString(element[0])
                    #
                    dataToSave.append(newDoc)
                # ---end loop of days---
            # ---end loop of portfolio---
            # print("Save Factors", factorName)
            database.Upsert_Many("Portfolio", "Factor_Portfolio", [], dataToSave)

    #
    pathname = "D:/Data/FactorAnalysis/"

    # build Cache
    factors = {}
    localFactorParams = Factors.FactorParams()

    # 把传入参数补充到因子参数中
    for factorName, inputParam in factorParams.items():
        # local params 必须
        localParams = localFactorParams.get(factorName)
        factors[factorName] = localParams
        # input params 补充
        for paramName, paramValue in inputParam.items():
            factors[factorName][paramName] = paramValue

    #
    factorNames = list(factors.keys())

    # ---Factor List / Aka Positions---
    # dailyFactor = []
    # monthlyFactor = []
    # quarterlyFactor = []
    # yearlyFacttor = []
    # halfYearFactor = []
    for factorName in factorNames:
        # portfolio group 分组容器
        # factors[factorName]["Group"] = []
        if "Period" not in factors[factorName]:
            a = 1
        outputFolder = pathname + factorName + "/"
        Gadget.CreateFolder(outputFolder)

    # ---确定依赖时间---
    # datetime0 = datetime1
    # if len(quarterlyFactor) > 0:
    #     datetime0 = datetime1 + datetime.timedelta(days=-70)
    # elif len(monthlyFactor) > 0:
    #     datetime0 = datetime1 + datetime.timedelta(days=-32)

    datetime2 = Gadget.ToDate(datetime2) + datetime.timedelta(hours=24)

    #
    bmDataSeries = Gadget.GetTradingDays(database, datetime1, datetime2, backforword1Day=True)
    datetime1 = bmDataSeries[0]["DateTime"]
    datetime2 = bmDataSeries[-1]["DateTime"]

    #
    releaseDates = Gadget.GenerateReleaseDates(datetime1, datetime2)
    reportDates = Gadget.GenerateReportDates(datetime1, datetime2)

    #
    # ---Symbol Locator, 快速定位Symbol所属分组---
    symbolGroupLocator = {}
    dfReturnCache = pd.DataFrame()

    # ---loop days---
    lastDateTime = None
    # for bar in bmBarSeries:
    for dailyReturn in bmDataSeries:
        # curDateTime = bar["DateTime"]
        curDateTime = dailyReturn["DateTime"]
        print(curDateTime)

        # ---Exclude New Stock 计算因子组合时候排除新股---
        earlyDate = curDateTime + datetime.timedelta(days=-365)
        listedInstrument = Gadget.FindListedInstrument(database, earlyDate)
        dfSymbols = Gadget.InstrumentToDateFrame(listedInstrument)
        # print(dfSymbols.head())

        # ---Prepare Daily Return 当天所有Returns---
        dfReturns = ReturnsToDataframe(database, curDateTime)
        dfReturns = dfReturns[(dfReturns["Return"] >= -0.11) & (dfReturns["Return"] <= 0.11)]
        # print(dfReturns.head())

        # ---Prepare DailyReturn Cache---
        dfTempReturns = dfReturns.set_index(["Symbol"])
        dfTReturns = dfTempReturns.T
        dfTReturns["DateTime"] = curDateTime
        dfTReturns.set_index(["DateTime"], inplace=True)
        # print(dfTReturns.head())
        #
        dfReturnCache = pd.concat([dfReturnCache, dfTReturns], axis=0, sort=False)
        # dfReturnCache.to_csv("d://a.csv")
        a = 0

        # ---Update Returns---
        for factorName in factorNames:
            data = factors[factorName]
            UpdateValue_Portfolio(data, dfReturns, curDateTime)
            pass

        # ---是否是调仓日？---
        # ---Find Rebalance Data 确定调仓日期---
        monthChange = False
        quarterChange = False
        #
        if lastDateTime != None and curDateTime.month != lastDateTime.month:
            monthChange = True
        #
        curReleaseDate = None
        for releaseDate in releaseDates:
            if lastDateTime != None and lastDateTime < releaseDate and curDateTime >= releaseDate:
                quarterChange = True
                curReleaseDate = releaseDate
                break
        #
        reportDateChange = False
        curReportDate = None
        for reportDate in reportDates:
            if lastDateTime != None and lastDateTime < reportDate and curDateTime >= reportDate:
                reportDateChange = True
                curReportDate = reportDate
                break

        # ---Rebalance based every factors 确定该因子是否需要调仓---
        # 分因子处理
        for factorName in factorNames:
            params = factors[factorName]
            groupType = params.get("GroupType")
            if groupType == None:
                groupType = "Percentile"

            outputFolder = pathname + factorName + "/"
            rebalance = False

            # ---是否需要 Rebalance---
            if factors[factorName]["Period"] == "Daily":
                rebalance = True
            if monthChange and factors[factorName]["Period"] == "Monthly":
                rebalance = True
                print("Monthly Change Rebalance", curDateTime)
            if quarterChange and factors[factorName]["Period"] == "Quarterly":
                rebalance = True
                print("Quarter Change Rebalance", curDateTime)
            if reportDateChange and factors[factorName]["Period"] == "ReportDate":
                rebalance = True
                print("ReportDate Change Rebalance", curDateTime)

            # ---初始化---
            if factors[factorName].get("Portfolio") == None:
                Initialize_Portfolio(factors, factorName, groupType)
                rebalance = True
                print("Initialize Rebalance", curDateTime)

            # ---Rebalance---
            if rebalance:
                # ---Calculate Single Stock Return---
                if factors[factorName]["RebalanceDate"] != None:
                    CalculateRangeReturn(dfReturnCache, factors[factorName]["Portfolio"], factors[factorName]["RebalanceDate"], curDateTime)

                # --- Profile Factor Value---
                p = factors[factorName]["Period"]
                dfFactor = Analysis.General.Profile(database, curDateTime, [factorName])

                # ---Delete Nan, inf, Special Process(Non-Negative)---
                # print(dfFactor.head())
                dfFactor = Factors.ProcessFactors(dfFactor)
                # print(dfFactor.head())

                if len(dfFactor) < 100:
                    # print("Missing Factor", factorName)
                    continue

                # ---排除新股---
                dfFactor = pd.merge(dfSymbols, dfFactor, how="inner", on="Symbol")
                # print(dfFactor)
                if output:
                    dfFactor.to_csv(outputFolder + "Profile_" + Gadget.ToDateString(curDateTime) + ".csv")
                #
                Rebalance_Portfolio(factors, factorName, dfFactor, groupType, curDateTime)
                pass

        # End Loop Factors
        #
        lastDateTime = curDateTime
    # End of Loop Dates

    # Finalize Calculate Stock Return
    for factorName, value in factors.items():
        t1 = factors[factorName]["RebalanceDate"]
        t2 = curDateTime
        CalculateRangeReturn(dfReturnCache, factors[factorName]["Portfolio"], t1, t2)

    # ---Export---
    Export_Portfolio_NetAsset(factors, datetime1, datetime2)

    # ---Save to Database--
    if save:
        Save_Portfolio(database, factors)

    return factors


# groupType = "HiLo",
# HiLo(343), Percentile, Quartile
def BuildModel_Multifactors(database, datetime1, datetime2, factors,
                            market="000001.SH",
                            update=False,
                            save=False,
                            callback=None):
    #
    def Automatic_BuildModel(database, datetime1, datetime2, factorName, param):
        #
        filter = {}
        filter["DateTime"] = {">=": datetime1, "<=": datetime2}
        filter["FactorName"] = factorName
        filter["GroupType"] = param.get("GroupType")
        #
        hiDocuments = database.Find("Portfolio", "Factor_Portfolio", filter)
        #
        if len(hiDocuments) == 0:
            # No Data
            lackData = True
            print("Missing Data or Force to Update", factorName, datetime1, datetime2)
            BuildFactorPortfolio(database, datetime1, datetime2, {factorName: param}, update)
        else:
            # Check if missing
            dfHi = Gadget.DocumentsToDataFrame(hiDocuments, keep=["Date", "Value"])
            storedDatetime1 = dfHi.iloc[0]["Date"]
            storedDatetime2 = dfHi.iloc[-1]["Date"]

            # if missing 数据不足即重新处理
            if datetime1 < storedDatetime1:
                lackData = True
                print("Missing Data", factorName, datetime1, storedDatetime1)
                BuildFactorPortfolio(database, datetime1, storedDatetime1, {factorName: param}, update)
            elif storedDatetime2 > datetime2:
                lackData = True
                print("Missing Data", factorName, storedDatetime2, datetime2)
                BuildFactorPortfolio(database, storedDatetime2, datetime2, {factorName: param}, update)

    #
    dfTrandingDays = Gadget.GetTradingDays_DataFrame(database, datetime1, datetime2, backforword1Day=False)
    # print(dfTrandingDays)
    datetime1 = dfTrandingDays.iloc[0]["Date"]
    datetime2 = dfTrandingDays.iloc[-1]["Date"]

    #
    dfFactors = pd.DataFrame()

    # ---load benchmark---
    if market != None:
        bmSymbol = market
        dfMarket = General.PeriodReturn_DataFrame(database, bmSymbol, datetime1, datetime2, instrumentType="Index")
        dfMarket.rename(columns={bmSymbol: "Market"}, inplace=True)
        dfFactors = dfMarket
        # print(dfMarket.dtypes)
        # print(dfMarket)

    # Check FactorPortfolio if existed
    # Build it, if not
    factorNames = list(factors.keys())
    for factorName in factorNames:
        param = factors[factorName]
        groupType = param["GroupType"]
        #
        if groupType == "Percentile":
            lastGroup = 9
        elif groupType == "HiLo":
            lastGroup = 2
        else:
            print("Unrecognized", groupType)
            continue
        #
        filter = {}
        filter["Date"] = {">=": datetime1, "<=": datetime2}
        filter["FactorName"] = factorName
        filter["GroupType"] = groupType

        if save:
            if update:
                # 强制计算
                BuildFactorPortfolio(database, datetime1, datetime2, {factorName: param}, save=True)
            else:
                Automatic_BuildModel(database, datetime1, datetime2, factorName, param)
            #
            filter["Number"] = 0
            hiDocuments = database.Find("Portfolio", "Factor_Portfolio", filter)
            dfHi = Gadget.DocumentsToDataFrame(hiDocuments, keep=["Date", "Value"])
            #
            filter["Number"] = lastGroup
            hiDocuments = database.Find("Portfolio", "Factor_Portfolio", filter)
            dfLo = Gadget.DocumentsToDataFrame(hiDocuments, keep=["Date", "Value"])

        else:
            # 直接计算
            tmpFactors = BuildFactorPortfolio(database, datetime1, datetime2, {factorName: param}, save=False)
            a = tmpFactors[factorName]["Portfolio"]
            #
            documentsHi = a[0]["DataSeries"]
            dfHi = pd.DataFrame(data=documentsHi, columns=["Date", "Hi"])
            dfHi["Date"] = dfHi["Date"].dt.date
            #
            documentsLo = a[-1]["DataSeries"]
            dfLo = pd.DataFrame(data=documentsLo, columns=["Date", "Lo"])
            dfLo["Date"] = dfLo["Date"].dt.date
        #
        dfHi.rename(columns={"Value": "Hi"}, inplace=True)
        dfLo.rename(columns={"Value": "Lo"}, inplace=True)
        df = pd.merge(dfHi, dfLo, on="Date", how="left")
        df[factorName] = df["Hi"] - df["Lo"]
        df = df[["Date", factorName]]
        # print(df)
        #
        if dfFactors.empty:
            dfFactors = df
        else:
            # print(dfFactors.dtypes)
            # print(df.dtypes)
            dfFactors = pd.merge(dfFactors, df, on="Date", how="left")

    # print(dfFactors)
    # ---Export---
    return dfFactors


def ProcessFields(dfFactors):
    #
    factorNames = list(dfFactors.columns)
    if "DateTime" in factorNames:
        factorNames.remove("DateTime")
    if "Date" in factorNames:
        factorNames.remove("Date")
    toDelete = []
    for factorName in factorNames:
        if "Unnamed" in factorName:
            toDelete.append(factorName)
    for d in toDelete:
        factorNames.remove(d)

    return factorNames


def RegressToFactorModel(dfPortfolio, dfFactors, print=True, plot=False):

    # print(dfPortfolio)
    # print(dfFactors)
    # dfFactors["DateTime"] = pd.to_datetime(dfFactors["DateTime"])
    # dfPortfolio["DateTime"] = pd.to_datetime(dfPortfolio["DateTime"])
    # print(dfPortfolio.dtypes)
    # print(dfFactors.dtypes)

    # prepare Columns
    factorNames = ProcessFields(dfFactors)
    #
    mergeField = "DateTime"
    if "Date" in dfPortfolio.columns:
        mergeField = "Date"
    df = pd.merge(dfPortfolio, dfFactors, on=mergeField, how="left")
    # df.drop(columns="DateTime", inplace=True)
    if mergeField == "Date":
        df.rename(columns={"Date": "DateTime"}, inplace=True)
    df.dropna(inplace=True)
    # print(df)

    # Runmodel
    model = Analysis.RegressionAnalysis.Regression_SM(df[factorNames], df[["Portfolio"]], printResult=print)

    output = pd.DataFrame(index=factorNames, columns=['coef', 'pvalue'])
    for x_name in factorNames:
        output.loc[x_name, 'coef'] = model.params[x_name]
        output.loc[x_name, 'pvalue'] = model.pvalues[x_name]

    if print:
        print(model.summary())
        print("Mean of Residual", np.mean(model.resid))

    if plot:
        plt.figure()
        plt.subplot(211)

        # ols.fit().model is a method to access to the residual.
        model.resid.plot.density()
        #plt.show()

        # plot Residuals
        plt.subplot(212)
        #model.resid.plot()
        plt.scatter(df.DateTime, model.resid)
        plt.show()

        #
        # plt.figure(figsize=(20, 10))
        # plt.scatter(df.spy, simple.resid)
        # plt.axhline(0.05)
        # plt.axhline(-0.05)
        # plt.xlabel('x value')
        # plt.ylabel('residual')
        # plt.show()
    #
    return model


def TopPortfolio(database, datetime1, datetime2, start=0, end=100):

    # ---Exclude New Stock---
    stdDatetime = Gadget.ToUTCDateTime(datetime1)
    earlyDate = stdDatetime + datetime.timedelta(days=-365)
    listedInstrument = Gadget.FindListedInstrument(database, earlyDate, datetime2)

    #
    data = []
    #
    filter = {}
    filter["StdDateTime"] = {}
    filter["StdDateTime"]["$gte"] = datetime1
    filter["StdDateTime"]["$lte"] = datetime2
    i = 0
    for instrument in listedInstrument:
        symbol = instrument["Symbol"]
        filter["Symbol"] = symbol
        i += 1
        #if i > 100:
        #    continue
        #
        if i % 100 == 0:
            print("Process", i)
        #
        barSeries = database.Find("Stock", "DailyBar", filter)
        if len(barSeries) == 0:
            continue
        bar1 = barSeries[0]
        bar2 = barSeries[-1]
        rangeReturn = bar2["Values"]["BClose"] / bar1["Values"]["BClose"] - 1
        data.append([symbol, rangeReturn])
    #
    df = pd.DataFrame(data, columns=["Symbol", "Return"])
    df.sort_values(by="Return", ascending=False, inplace=True)
    df = df[start:end]
    #print(df)

    return df


def GeneratePortfolioDataFrame(database, symbols, datetime1, datetime2):
    #
    bmSymbol = "000001.SH"
    df = IO.LoadBarsAsDataFrame(database, bmSymbol, datetime1, datetime2, ["Close"], instrumentType="Index")
    df.drop(columns="StdDateTime", inplace=True)
    df.rename(columns={'Close': bmSymbol}, inplace=True)
    # print(df)

    #
    count = 0
    totalCount = len(symbols)
    for symbol in symbols:
        count += 1
        if count % 100 == 0:
            print("Load Multi-Instruments Bars As DataFrame " + str(count) + " / " + str(totalCount))

        #
        tempDf = IO.LoadBarsAsDataFrame(database, symbol, datetime1, datetime2, ["BClose"], instrumentType="Stock")
        tempDf.drop(columns="StdDateTime", inplace=True)
        tempDf.rename(columns={'BClose': symbol}, inplace=True)
        if tempDf.empty:
            continue

        # ---First time loop---
        if df.empty:
            df = tempDf
        else:
            df = pd.merge(df, tempDf, on='DateTime', how='left')

    #
    df.drop(columns=bmSymbol, inplace=True)
    for symbol in symbols:
        df[symbol + "_Return"] = df[symbol] / df[symbol].shift(1) - 1
        df.drop(columns=symbol, inplace=True)
    #
    df.fillna(method="ffill", inplace=True)
    # 是为了求和方便
    df.set_index("DateTime", inplace=True)
    #print(df)
    df['Portfolio'] = df.apply(lambda x: x.mean(), axis=1)
    df.reset_index(inplace=True)

    # ---只需保留两列作为结果---
    df = df[["DateTime","Portfolio"]]
    # print(df)

    #
    return df


def Top100Portfolio(database, datetime1, datetime2, fromIndex=0, toIndex=100):
    #
    dfTop = TopPortfolio(database, datetime1, datetime2, start=fromIndex, end=toIndex)
    dfTop.to_csv("D:/Data/FactorModel/" + "Top.csv")
    #
    symbols = list(dfTop["Symbol"])
    dfPortfolio = GeneratePortfolioDataFrame(database, symbols, datetime1, datetime2)
    #
    return dfPortfolio


def BotPortfolio(database, datetime1, datetime2, fromIndex=0, toIndex=100):
    pass


def FactorPortfolio(database, datetime1, datetime2, factorName,
                    rebanlance,  # Daily, Monthly, Quarterly, Yearly
                    groupPercentage,  # 0.1 0.3 0.5
                    groupType):  # Percentile, Q
    pass


def StepWiseRegressToFactorModel(dfPortfolio, dfFactors):
    #
    factorNames = ProcessFields(dfFactors)

    #
    df = pd.merge(dfPortfolio, dfFactors, on="DateTime", how="left")
    df.dropna(inplace=True)
    # print(df)

    result = Analysis.RegressionAnalysis.StepwiseRegression(df[factorNames], df["Portfolio"])
    print(result)
    a = 0



# Carhart-4

# 对行业指数进行归因
def IndustryIndexRegression():
    dfIndex = pd.read_csv("D:/Data/IndustryAnalysis/IndustryIndex/" + "家电_EquallyWeightIndex.csv", engine="python")
    dfPortfolio = dfIndex[["DateTime", "Mean"]]
    dfPortfolio = dfPortfolio.rename(columns={"Mean":"Portfolio"})
    dfPortfolio.dropna(inplace=True)
    print(dfPortfolio)
    dfFF3Model = pd.read_csv("D:/Data/FactorModel/" + "FamaFrench3_Total.csv")
    RegressToFactorModel(dfPortfolio, dfFF3Model)


def CreateTopPortfolio():

    #for year in

    #dfPortfolio = Top100Portfolio(database, datetime1, datetime2)
    #dfPortfolio.to_csv("D:/Data/FactorModel/" + "dfPortfolio_2016.csv")

    #earlyDate = stdDatetime + datetime.timedelta(days=-365)
    #listedInstrument = Gadget.FindListedInstrument(database, earlyDate, datetime2)
    pass


if __name__ == '__main__':
    #
    from Core.Config import *
    config = Config()
    database = config.DataBase("MySQL")
    realtime = config.RealTime()

    # ---Infrastructure---
    #database.creatIndex("Misc", "Model", "DateTime")
    #database.creatIndex("Misc", "Model", "Factor")
    #database.creatIndex("Misc", "Model", "Params")
    #
    # database.creatIndex("Misc", "FactorPortfolio", "DateTime")
    # database.creatIndex("Misc", "FactorPortfolio", "Factor")
    # database.creatIndex("Misc", "FactorPortfolio", "Group")

    # ---Build Model---
    datetime1 = datetime.datetime(2017, 1, 1)
    datetime2 = datetime.datetime(2019, 12, 13)

    # Build Fama-3 Model
    dfFF3Model = BuildModel_FamaFrench3(database, datetime1, datetime2, update=False, save=True)
    print(dfFF3Model)
    # sDateTime = Gadget.ToDateString(datetime2)
    # dfFF3Model.to_csv("D:/Data/FactorModel/" + "FamaFrench3_" + sDateTime + ".csv")

    # Build MultiFactor Model
    dfTotalDFactorModel = BuildModel_MultiFators_TotalFactors(database, datetime1, datetime2, saveToDatebase=False)
    # print(dfTotalDFactorModel)
    # dfTotalDFactorModel.to_csv("D:/Data/FactorModel/" + "TotalFactor_" + sDateTime + ".csv")

    # ---Performance Attribute---
    IndustryIndexRegression()

    # ---风格测试---
    # dfPortfolio = Top100Portfolio(database, datetime1, datetime2)
    # dfPortfolio.to_csv("D:/Data/FactorModel/" + "dfPortfolio_2016.csv")
    # dfPortfolio = pd.read_csv("D:/Data/FactorModel/" + "dfPortfolio_2019Q1.csv")
    # ---Common Regression---
    # RegressToFactorModel(dfPortfolio, dfFactorModel)
    # StepWiseRegressToFactorModel(dfPortfolio, dfFactorModel)

    # ---随意拼接单因子模型测试---
    datetime1 = datetime.datetime(2017,2,1)
    datetime2 = datetime.datetime(2017,3,1)
    sDateTime = Gadget.ToDateString(datetime2)
    #
    factors = ["Cap", "PB_LF"]
    factors.append("ProfitMargin_OperatingProfit1_LYR")
    factors.append("ProfitMargin_OperatingProfit_LYR")
    factors.append("ProfitMargin_OperatingProfit1")
    factors.append("ProfitMargin_OperatingProfit")
    factors.append("ROE_NetIncome2_TTM")
    factors.append("ROA_NetIncome2_TTM")
    factors.append("ROE_NetIncome2_LYR")
    factors.append("ROA_NetIncome2_LYR")

    dfFactorModel = LoadModel(database, datetime1, datetime2, factors)
    # dfFactorModel.to_csv("D:/Data/FactorModel/" + "temp.csv")
    # dfFactorModel = pd.read_csv("D:/Data/FactorModel/" + "temp.csv")
    # dfFactorModel = pd.read_csv("D:/Data/FactorModel/" + "FamaFrench3_Total.csv")

    # ---Save Model---
    # dfFF3Model = pd.read_csv("D:/Data/FactorModel/" + "FamaFrench3_Total.csv")
    # dfFF3Model.drop(columns=["Unnamed: 0"],inplace=True)
    # dfFF3Model["DateTime"] = pd.to_datetime(dfFF3Model["DateTime"])
    # SaveModel(dfFF3Model, "FamaFrench3")
    # RegressToFactorModel(dfPortfolio, dfFF3Model)


    # database = MongoDB.MongoDB("Localhost","27017")
    # sort = [("StdDateTime",pymongo.ASCENDING)]

    # TestPortfolio(database)
    # Portfolio.MovingMixDatabaseToSeperate(database)

    # FamaFrench3_PrepareData(database)
    # FamaFrench3_BuildModel(database)

    # database.delete("Portfolio","Fills",{"Portfolio":"SmallValue"})
    # IO.ExportDataBaseResult(database,"Portfolio","Fills",{"Portfolio":"SmallValue"},["Symbol","CashFlow","Price","Qty","StdDateTime"],"D://Data//FamaFrench3//Fills.csv")